# Copyright (c) 2009 Lemur Consulting Ltd.
# 
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:

# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.

import os.path
import sys
import xapian
import search_timer


db = xapian.flint_open('xapian.db', xapian.DB_CREATE)
db.begin_transaction()

fcount = 0

tg = xapian.TermGenerator()

def index_dir(arg, dirpath, namelist):
    global fcount
    
    search_timer.start();
    for name in namelist:
        if name.endswith('.txt'):
            file = open(os.path.join(dirpath, name))
            data = file.read()            
            file.close()
            doc = xapian.Document()
            tg.set_document(doc)
            tg.index_text_without_positions(data)            
            db.add_document(doc)
            fcount += 1

            if fcount % 1000 == 0:
                print fcount
                
            if fcount % 10000 == 0:
                db.commit_transaction()
                db.begin_transaction()
            
os.path.walk(sys.argv[1], index_dir, None)
db.commit_transaction()

search_timer.stop();
print "indexing took %s microseconds" % search_timer.micro()
